package test;

import cn.tyoui.httpclient.HttpCrawler;

import java.io.BufferedReader;
import java.io.FileReader;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Stream;

public class CrawlerArea {
    public static void main(String[] args) throws Exception {
        String file = "D:\\ideaCode\\Crawler\\src\\main\\java\\test\\shopArea.txt";
        BufferedReader bufferedReader = new BufferedReader(new FileReader(file));
        List<String> list = new ArrayList<>();
        while (bufferedReader.ready()) {
            Stream<String> stream = bufferedReader.lines();
            stream.forEach(s -> {
                String[] strings = s.split("'");
                for (int i = 1; i <= Integer.parseInt(strings[1]); i++) {
                    list.add(strings[0] + "p" + i);
                }
            });
        }
        HttpCrawler httpCrawler = new HttpCrawler();
        httpCrawler.setDir("D://shopArea");
        for (String s : list) {
            httpCrawler.startCrawler(s, 5, 10);
        }
    }
}
